#!/bin/sh
#/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#/* Copyright 2002-2003 by Mark Wolgemuth, All rights reserved.             */
#/* Copyright 2002-2003 by Employease, Inc.  All rights reserved.           */
#/*                                                                         */
#/* Redistribution and use in source and binary forms, with or without      */
#/* modification, are permitted provided that the following conditions      */
#/* are met:                                                                */
#/*                                                                         */
#/*    1. Redistributions of source code must retain the above copyright    */
#/* notice, this list of conditions and the following disclaimer.           */
#/*    2. Redistributions in binary form must reproduce the above copyright */
#/* notice, this list of conditions and the following disclaimer in the     */
#/* documentation and/or other materials provided with the distribution.    */
#/*                                                                         */
#/* THIS SOFTWARE IS PROVIDED BY THE MOD_ATHENA PROJECT ``AS IS'' AND ANY   */
#/* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE       */
#/* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR      */
#/* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FREEBSD PROJECT OR        */
#/* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,   */
#/* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,     */
#/* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR      */
#/* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY     */
#/* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT            */
#/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   */
#/* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.    */
#/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

ATH_AGENT_SH_VERSION="ath_monitor.sh 0.2 2004/05/12 (C) mark wolgemuth"

## search down to "main begin" for logic flow
##

## internal vars
if [ "${ATH_ETC_DIR}" = "" ]
then
    ATH_ETC_DIR="."
fi
. ${ATH_ETC_DIR}/ath_shell_cmds.sh

ATH_PID_FILE=/tmp/ath_monitor_sh.pid
GET_CMD=""
AGENT_OPTS=""
HOST_TYPE=`uname`
ATH_OPT_VERBOSE=""
ATH_OPT_QUIET=""
ATH_OPT_BG="NO"
ATH_OPT_LOG=""
ATH_OPT_BURL=""
ATH_OPT_FREQ=""
ATH_OPT_HOST=""
ATH_OPT_LOGFILE=""
ATH_OPT_EMAIL_ADDRS=""
ATH_OPT_RESEND=""
ATH_OPT_PROBE="0"
ATH_OPT_PROBE_AGAIN="0"
ATH_PROBE_PASS="1"
ATH_PHYS_LIST=""
ATH_ALERT_SUBJECT="ALERT: HOST DOWN"
ATH_TIMEDOUT_LIST=""
ATH_DOWN_LIST=""
# SICK = TIMEDOUT or DOWN
ATH_SICK_LIST=""
ATH_SICK_LIST_LAST=""
ATH_SICK_LIST_LAST_TIME=""
SEP="|"

## function defs

# usage() prints usage
usage() {
    echo "ath_monitor.sh [-v] [-q] [-b] [-u BASEURL] [-f frequency]"
    echo "  [-m email] [-r resend delay] [-p] [-P] [-k]"
    echo ""
    echo "  -v       Prints version and exits"
    echo "  -q       Suppresses all status and warning messages (quiet)"
    echo "  -b       Runs in background"
    echo "  -u (url) Specifies base URL to reach the athena engine"
    echo "  -f (sec) Sets monitor frequency (default to 5)"
    echo "  -m (eml) Sets email address(es) to send errors alerts to"
    echo "  -r (sec) Resend alert every (x) seconds, never if 0 (default), always if -1"
    echo "  -p       Probe server for currently \"online\" machines, and only check those (default checks all)"
    echo "  -P       Re-probe in every loop run, in case online/offline has changed"
    echo "  -k       Kill the currently running daemon and exit"
    return 0
}

# set_opts() runs getopts and sets defaults
set_opts() {
    while getopts "vVqblu:f:o:m:r:pPk" OPTTYPE; do
	case ${OPTTYPE} in
	"v")
	    echo "${ATH_AGENT_SH_VERSION}"
	    exit
	    ;;
	"V") 
	    ATH_OPT_VERBOSE="1"
	    ;;
	"q")
	    ATH_OPT_QUIET="1"
	    ;;
	"b")
	    ATH_OPT_BG="YES"
	    ;;
	"l")
	    ATH_OPT_LOG="1"
	    ;;
	"u")
	    ATH_OPT_BURL="${OPTARG}"
	    ;;
	"f")
	    ATH_OPT_FREQ="${OPTARG}"
	    ;;
	"o")
	    ATH_OPT_LOGFILE="${OPTARG}"
	    touch ${ATH_OPT_LOGFILE} > /dev/null 2>&1
	    if !(test -e ${ATH_OPT_LOGFILE}); then
		echo "Cannot create log file: ${ATH_OPT_LOGFILE}"
	    fi
	    ATH_OPT_LOG="1"
	    ;;
	"m")
	    ATH_OPT_EMAIL_ADDRS=`echo ${OPTARG} | sed -e 's/,/ /g'`
	    ;;
	"r")
	    ATH_OPT_RESEND="${OPTARG}"
	    ;;
	"p")
	    ATH_OPT_PROBE="1"
	    ;;
	"P")
	    ATH_OPT_PROBE_AGAIN="1"
	    ;;
	"k")
	    loop_kill
	    exit
	    ;;
	[?])
	    usage
	    exit
	    ;;
	esac
    done
    # do defaults
    if [ "${ATH_OPT_BURL}" = "" ]; then
	status_msg "WARNING: using default base athena engine URL"
	ATH_OPT_BURL="http://localhost:8080/ath"
    fi
    if [ "${ATH_OPT_HOST}" = "" ]; then
	ATH_OPT_HOST=`hostname`
    fi
    if [ "${ATH_OPT_FREQ}" = "" ]; then
	ATH_OPT_FREQ="5"
    fi
    if [ "${ATH_OPT_UPDATES}" = "" ]; then
	ATH_OPT_UPDATES="cpu"
    fi
    return 0
}

probe_run() {
    # get it to return "PASS" in response so we know its working
    GET_CMD="${AGENT_DIR}/${AGENT_BIN} ${AGENT_OPTS} \
	${ATH_OPT_BURL}/status/phys?h=*,f=PASS${SEP}%h${SEP}%o"
    if [ "${ATH_OPT_VERBOSE}" = "1" ]; then
	status_msg $GET_CMD
    fi
    GET_RESULT=`$GET_CMD`
    for PHYS in $GET_RESULT
    do
	ATH_WORKS=`echo ${PHYS}| awk -F "${SEP}" '{ print $1; }'`;
	PHYS_HOST=`echo ${PHYS}| awk -F "${SEP}" '{ print $2; }'`;
	PHYS_ON=`echo ${PHYS}| awk -F "${SEP}" '{ print $3; }'`;
	if [ "$ATH_WORKS" = "PASS" ]
	then
	    if [ "$PHYS_ON" = "1" ]
 	    then
		ATH_PHYS_LIST="$PHYS_HOST $ATH_PHYS_LIST"
	    fi
	else
	    status_msg "ERROR: SERVER ERROR"
	fi
    done
    if [ "$ATH_PHYS_LIST" = "" ]
    then
	status_msg "WARNING: There are no hosts up on this engine"
    else
	status_msg "Checking probed hosts: $ATH_PHYS_LIST"
    fi
}
	
loop_run() {
    if [ "$ATH_OPT_PROBE_AGAIN" = "1" ]
    then
	ATH_PHYS_LIST=""
	ATH_OPT_PROBE="1"
	probe_run
    fi
    GET_CMD="${AGENT_DIR}/${AGENT_BIN} ${AGENT_OPTS} \
	${ATH_OPT_BURL}/status/phys?h=*,f=PASS${SEP}%h${SEP}%u${SEP}%i"
    if [ "${ATH_OPT_VERBOSE}" = "1" ]; then
	status_msg $GET_CMD
    fi
    GET_RESULT=`$GET_CMD`
    for PHYS in $GET_RESULT
    do
	ATH_WORKS=`echo ${PHYS}| awk -F "${SEP}" '{ print $1; }'`;
	PHYS_HOST=`echo ${PHYS}| awk -F "${SEP}" '{ print $2; }'`;
	# marked up or down
	PHYS_UP=`echo ${PHYS}| awk -F "${SEP}" '{ print $3; }'`;
	# timed in or out
	PHYS_IN=`echo ${PHYS}| awk -F "${SEP}" '{ print $4; }'`;
	if [ "$ATH_WORKS" = "PASS" ]
	then
	    if [ "$ATH_OPT_PROBE" = "1" ]
	    then
		for PHYS_IN_LIST in $ATH_PHYS_LIST
		do
		    if [ "$PHYS_HOST" = "$PHYS_IN_LIST" ]
		    then
			ATH_PROBE_PASS="1"
			break
		    fi
		done
	    fi
	    if [ "$ATH_OPT_PROBE" = "0" -o "$ATH_PROBE_PASS" = "1" ]
	    then
		if [ "$PHYS_UP" = "1" ]
		then
		    # host is up
		elif [ "$PHYS_UP" = "0" ]
		then
		    ATH_DOWN_LIST="$PHYS_HOST $ATH_DOWN_LIST"
		else
		    status_msg "ERROR: $PHYS_HOST INVALID RESPONSE"
		fi
		if [ "$PHYS_IN" = "1" ]
		then 
		    # host is in
		elif [ "$PHYS_IN" = "0" ]
		then
		    ATH_TIMEDOUT_LIST="$PHYS_HOST $ATH_TIMEDOUT_LIST"
		else
		    status_msg "ERROR: $PHYS_HOST INVALID RESPONSE"
		fi
	    fi
	else
	    status_msg "ERROR: REMOTE SERVER ERROR"
	fi
	ATH_PROBE_PASS="0"
    done
    DATETIME=`date`
    if [ ! "$ATH_DOWN_LIST" = "" ]
    then
	ATH_SICK_LIST="$ATH_DOWN_LIST"
    fi
    if [ ! "$ATH_TIMEDOUT_LIST" = "" ]
    then
	if [ ! "$ATH_SICK_LIST" = "" ]
	then
	    ATH_SICK_LIST="$ATH_SICK_LIST $ATH_TIMEDOUT_LIST"
	else
	    ATH_SICK_LIST="$ATH_TIMEDOUT_LIST"
	fi
    fi
    # dirty, but this is shell. empty will contain space due to above
    if [ ! "$ATH_SICK_LIST" = "" ]
    then
	if [ "$ATH_OPT_EMAIL_ADDRS" = "" ]
	then
	    status_msg "No emails to alert."
	else
	    if [ ! "$ATH_OPT_RESEND" = "" ]
	    then
		if [ "$ATH_SICK_LIST" = "$ATH_SICK_LIST_LAST" ]
		then
		    #handle resend rules
		    CUR_UNIXTIME=`unixtime`
		    TIME_SLIP=`expr $CUR_UNIXTIME - $ATH_SICK_LIST_LAST_TIME`
		    if [ "$TIME_SLIP" -gt "$ATH_OPT_RESEND" ]
		    then 
			ATH_SEND_EMAIL="1"
			ATH_SICK_LIST_LAST_TIME="$CUR_UNIXTIME"
			status_msg "Resend window exceeded, resending."
		    else
			ATH_SEND_EMAIL=""
		    fi
		else
		    ATH_SICK_LIST_LAST="$ATH_SICK_LIST"
		    ATH_SICK_LIST_LAST_TIME=`unixtime`
		    ATH_SEND_EMAIL="1"
		fi
	    else
		ATH_SEND_EMAIL="1"
	    fi
	    if [ "$ATH_SEND_EMAIL" = "1" ]
	    then
		status_msg "Sending alert to: ${ATH_OPT_EMAIL_ADDRS}"
		echo -e "To: ${ATH_OPT_EMAIL_ADDRS}\nSubject: ${ATH_ALERT_SUBJECT}\n\nHosts down: ${ATH_DOWN_LIST}\nHosts timedout: ${ATH_TIMEDOUT_LIST}\nLB url: ${ATH_OPT_BURL}\nDate: ${DATETIME}\n\n.\n" | $ATH_SENDMAIL_BIN
	    fi
	fi
	if [ ! "$ATH_DOWN_LIST" = "" ]
	then
	    status_msg "${DATETIME}: Hosts down: ${ATH_DOWN_LIST}"
	fi
	if [ ! "$ATH_TIMEDOUT_LIST" = "" ]
	then
	    status_msg "${DATETIME}: Hosts timed out: ${ATH_TIMEDOUT_LIST}"
	fi
    fi
    ATH_DOWN_LIST=""
    ATH_TIMEDOUT_LIST=""
    ATH_SICK_LIST=""
}

## main begin ##
################

setup
set_opts $@
chk_agent_bin
add_agent_opts
if [ "$ATH_OPT_PROBE" = "1" ]
then
    probe_run
fi
exec_handler

################
### main end ###
